#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-10-22 21:06:48
# Project: m_autohome_com_cn_comment

import re
import json
import html
import time
import random
import string
import requests

from pyspider.libs.base_handler import *
from pyquery import PyQuery as pq

from src.mysqldb import SQL
from src.headers_switch import HeadersSelector
from src.utils import download_img
from src.utils import md5
from src.utils import oss
from src.utils import get_proxy


class Handler(BaseHandler):
    crawl_config = {
        'itag': 'v0.1.6',
        'proxy': '',
    }
    cookies = ''

    def get_cookies(self):
        cookie = {}
        return cookie
        for line in self.cookies.split(';'):
            key,value = line.split('=', 1) #1代表只分一次，得到两个数据
            cookie[key.strip()] =value.strip()
        return cookie

    @every(minutes=30)
    def on_start(self):
        self.crawl_config['proxy'] = get_proxy()
        header_slt = HeadersSelector()
        header = header_slt.select_header()  # 获取一个新的 header

        header['Host'] = ''
        #self.crawl('m.jd.com', callback=self.index_page)
        tag_li = [
            (21776, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "655163", "name": "u6cb9u8017u6ee1u610f", "num": "16"}, {"id": "368938", "name": "u5916u89c2u65f6u5c1a", "num": "9"}, {"id": "553022", "name": "u9694u97f3u4e0du597d", "num": "6"}, {"id": "524181", "name": "u60acu67b6u51cfu9707u786c", "num": "5"}, {"id": "590836", "name": "u524du6392u7a7au95f4u5927", "num": "4"}]),
            (23012, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "510756", "name": "u6cb9u8017u6ee1u610f", "num": "4"}, {"id": "345613", "name": "u540eu5907u53a2u7a7au95f4u7565u5c0f", "num": "4"}, {"id": "237999", "name": "u4e58u5750u7a7au95f4u8db3u591f", "num": "3"}, {"id": "285452", "name": "u540eu6392u7a7au95f4u5927", "num": "3"}]),
            (23358, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "463707", "name": "u4e58u5750u7a7au95f4u8212u9002", "num": "200"}, {"id": "298294", "name": "u6cb9u8017u6ee1u610f", "num": "153"}, {"id": "381213", "name": "u5916u89c2u6f02u4eae", "num": "128"}, {"id": "425213", "name": "u4f4eu914du914du7f6eu5f88u9ad8", "num": "81"}, {"id": "134594", "name": "u50a8u7269u7a7au95f4u5c11", "num": "61"}, {"id": "145300", "name": "u540eu6392u7a7au95f4u5bbdu655e", "num": "50"}, {"id": "154627", "name": "u80ceu566au6709u70b9u5927", "num": "50"}, {"id": "476765", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "45"}, {"id": "199036", "name": "u5473u9053u6709u70b9u5927", "num": "19"}, {"id": "143859", "name": "u5239u8f66u6709u70b9u8f6f", "num": "12"}]),
            (24733, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "402563", "name": "u6cb9u8017u6ee1u610f", "num": "42"}, {"id": "361739", "name": "u5916u89c2u6f02u4eae", "num": "36"}, {"id": "278109", "name": "u540eu6392u7a7au95f4u5c0f", "num": "29"}, {"id": "333804", "name": "u4e58u5750u7a7au95f4u5c0f", "num": "28"}, {"id": "249784", "name": "u50a8u7269u7a7au95f4u592au5c11", "num": "18"}, {"id": "297254", "name": "u540eu5907u53a2u7a7au95f4u4e0du9519", "num": "15"}, {"id": "314025", "name": "u8f66u5c3eu706fu5f88u6f02u4eae", "num": "15"}, {"id": "236148", "name": "u64cdu63a7u8f7bu677e", "num": "13"}, {"id": "315914", "name": "u8f6cu5411u7cbeu51c6", "num": "11"}, {"id": "257523", "name": "u60acu67b6u51cfu9707u504fu786c", "num": "8"}]),
            (25892, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "26905", "name": "u4e58u5750u7a7au95f4u4e0du9519", "num": "755"}, {"id": "101500", "name": "u5916u89c2u6f02u4eae", "num": "646"}, {"id": "188203", "name": "u6cb9u8017u6ee1u610f", "num": "620"}, {"id": "73177", "name": "u4f4eu914du914du7f6e", "num": "357"}, {"id": "58976", "name": "u5e95u76d8u4f4e", "num": "248"}, {"id": "167637", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "169"}, {"id": "131323", "name": "u5916u6f06u9762u592au8584", "num": "164"}, {"id": "14240", "name": "u50a8u7269u7a7au95f4u5f88u591a", "num": "159"}, {"id": "18027", "name": "u80ceu566au6709u70b9u5927", "num": "155"}, {"id": "144887", "name": "u9694u97f3u4e0du597d", "num": "136"}]),
            (26547, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "493866", "name": "u6cb9u8017u6ee1u610f", "num": "29"}, {"id": "370239", "name": "u4e58u5750u7a7au95f4u591fu7528", "num": "24"}, {"id": "219807", "name": "u5916u89c2u65f6u5c1a", "num": "13"}, {"id": "552056", "name": "u8f6cu5411u624bu611fu5f88u8f7b", "num": "7"}, {"id": "384076", "name": "u50a8u7269u7a7au95f4u5927", "num": "5"}, {"id": "285458", "name": "u5e95u76d8u4f4e", "num": "5"}, {"id": "439511", "name": "u60acu67b6u51cfu9707u786c", "num": "5"}, {"id": "268676", "name": "u8f6cu5411u4e0du9519", "num": "4"}, {"id": "209142", "name": "u540eu5907u53a2u7a7au95f4u5c0f", "num": "4"}, {"id": "385867", "name": "u9694u97f3u4e0du548bu6837", "num": "3"}]),
            (27310, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "138660", "name": "u4e58u5750u7a7au95f4u4e0du9519", "num": "191"}, {"id": "139756", "name": "u6cb9u8017u6ee1u610f", "num": "163"}, {"id": "139441", "name": "u5b89u5168u914du7f6eu4e30u5bcc", "num": "68"}, {"id": "140189", "name": "u9694u97f3u5dee", "num": "57"}, {"id": "244184", "name": "u5916u89c2u65f6u5c1a", "num": "48"}, {"id": "249267", "name": "u60acu67b6u51cfu9707u504fu786c", "num": "44"}, {"id": "139257", "name": "u540eu6392u7a7au95f4u5927", "num": "38"}, {"id": "227381", "name": "u540eu5907u53a2u7a7au95f4u591fu7528", "num": "35"}, {"id": "224820", "name": "u50a8u7269u7a7au95f4u592au5c11", "num": "34"}, {"id": "260267", "name": "u5916u6f06u9762u5f88u8584", "num": "28"}]),
            (28393, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "574333", "name": "u5916u89c2u65f6u5c1a", "num": "8"}, {"id": "593340", "name": "u4e58u5750u7a7au95f4u4e0du618bu5c48", "num": "6"}, {"id": "591490", "name": "u64cdu63a7u5f88u68d2", "num": "6"}, {"id": "620615", "name": "u6cb9u8017u6ee1u610f", "num": "4"}, {"id": "614462", "name": "u8f66u5c3eu706fu5f88u8d5e", "num": "3"}, {"id": "604246", "name": "u50a8u7269u7a7au95f4u633au5c0f", "num": "3"}, {"id": "590286", "name": "u60acu67b6u51cfu9707u6bd4u8f83u786c", "num": "3"}]),
            (29048, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "149591", "name": "u6cb9u8017u6ee1u610f", "num": "286"}, {"id": "9117", "name": "u5916u89c2u6f02u4eae", "num": "277"}, {"id": "9091", "name": "u4e58u5750u7a7au95f4u591fu7528", "num": "192"}, {"id": "9134", "name": "u5b89u5168u914du7f6eu8f83u4e3au6ee1u610f", "num": "124"}, {"id": "9162", "name": "u8f6cu5411u7cbeu51c6", "num": "120"}, {"id": "9111", "name": "u540eu6392u7a7au95f4u5c0f", "num": "114"}, {"id": "67463", "name": "u64cdu63a7u5f88u68d2", "num": "98"}, {"id": "29131", "name": "u60acu67b6u51cfu9707u504fu786c", "num": "91"}, {"id": "103566", "name": "u5916u6f06u9762u6709u70b9u8584", "num": "71"}, {"id": "15555", "name": "u9694u97f3u5dee", "num": "59"}]),
            (29316, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "9086", "name": "u4e58u5750u7a7au95f4u6ee1u610f", "num": "379"}, {"id": "163965", "name": "u6cb9u8017u6ee1u610f", "num": "232"}, {"id": "21893", "name": "u5916u89c2u7a33u91cd", "num": "211"}, {"id": "121764", "name": "u5b89u5168u914du7f6eu5f88u9ad8", "num": "145"}, {"id": "85239", "name": "u540eu5907u53a2u7a7au95f4u5927", "num": "131"}, {"id": "45977", "name": "u540eu6392u7a7au95f4u5927", "num": "93"}, {"id": "58941", "name": "u60acu67b6u51cfu9707u504fu786c", "num": "71"}, {"id": "80719", "name": "u5473u9053u6bd4u8f83u5927", "num": "55"}, {"id": "148279", "name": "u5012u8f66u5f71u50cfu5f88u4e0du597d", "num": "51"}, {"id": "25110", "name": "u80ceu566au6709u70b9u5927", "num": "33"}]),
            (29493, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "157207", "name": "u4e58u5750u7a7au95f4u6ee1u610f", "num": "116"}, {"id": "430621", "name": "u5916u89c2u6f02u4eae", "num": "53"}, {"id": "271486", "name": "u524du6392u5ea7u6905u8212u9002", "num": "27"}, {"id": "458018", "name": "u8f6cu5411u624bu611fu5f88u8f7b", "num": "23"}, {"id": "459311", "name": "u9694u97f3u597d", "num": "23"}, {"id": "317476", "name": "u540eu5907u53a2u7a7au95f4u5927", "num": "20"}, {"id": "181037", "name": "u5473u9053u592au91cd", "num": "13"}, {"id": "293990", "name": "u8d77u6b65u52a8u529bu6709u70b9u8089", "num": "11"}, {"id": "300803", "name": "u5185u9970u5851u6599u611fu5f3a", "num": "10"}, {"id": "278000", "name": "u4e2du63a7u53f0u6750u8d28u786c", "num": "3"}]),
            (30105, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "133385", "name": "u4e58u5750u7a7au95f4u8db3u591f", "num": "52"}, {"id": "133533", "name": "u5916u89c2u9738u6c14", "num": "46"}, {"id": "133649", "name": "u8f66u5185u9759u97f3u6548u679cu5f88u597d", "num": "34"}, {"id": "500366", "name": "u6cb9u8017u6ee1u610f", "num": "28"}, {"id": "481914", "name": "u5185u9970u914du7f6eu6ee1u610f", "num": "24"}, {"id": "296807", "name": "u97f3u54cdu771fu7684u5f88u68d2", "num": "16"}, {"id": "321978", "name": "u540eu5907u53a2u7a7au95f4u5927", "num": "15"}, {"id": "228293", "name": "u5e95u76d8u592au4f4e", "num": "15"}, {"id": "133476", "name": "u5012u8f66u5f71u50cfu5c71u5be8", "num": "11"}, {"id": "429375", "name": "u6276u624bu7bb1u4e0du5927", "num": "3"}]),
            (30975, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "663889", "name": "u4e58u5750u7a7au95f4u591fu7528", "num": "33"}, {"id": "709803", "name": "u6cb9u8017u6ee1u610f", "num": "33"}, {"id": "678249", "name": "u540eu5907u53a2u7a7au95f4u5927", "num": "15"}, {"id": "380713", "name": "u5b89u5168u914du7f6eu975eu5e38u9f50u5168", "num": "13"}, {"id": "703807", "name": "u5012u8f66u96f7u8fbeu6709u7528u4e0du4e0a", "num": "9"}, {"id": "693672", "name": "u50a8u7269u7a7au95f4u5f88u4e30u5bcc", "num": "8"}, {"id": "656957", "name": "u540eu6392u7a7au95f4u5f88u5927", "num": "8"}, {"id": "361283", "name": "u4e2du63a7u5c4fu592au5c0f", "num": "8"}, {"id": "620337", "name": "u9694u97f3u592au5dee", "num": "4"}, {"id": "137294", "name": "u80ceu566au4e0du5c0f", "num": "3"}]),
            (31040, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "135546", "name": "u4e58u5750u7a7au95f4u6ee1u610f", "num": "383"}, {"id": "200679", "name": "u5916u89c2u65f6u5c1a", "num": "281"}, {"id": "446100", "name": "u6cb9u8017u6ee1u610f", "num": "273"}, {"id": "445228", "name": "u5b89u5168u914du7f6eu6bd4u8f83u9ad8", "num": "230"}, {"id": "300099", "name": "u540eu5907u53a2u7a7au95f4u5c0f", "num": "141"}, {"id": "210717", "name": "u50a8u7269u7a7au95f4u5f88u591a", "num": "107"}, {"id": "447078", "name": "u8f6cu5411u7cbeu51c6", "num": "99"}, {"id": "446502", "name": "u7b2cu4e09u6392u7a7au95f4u7565u5c0f", "num": "40"}, {"id": "479863", "name": "u5473u9053u6bd4u8f83u5927", "num": "33"}, {"id": "232487", "name": "u53d1u52a8u673au566au97f3u6bd4u8f83u5927", "num": "30"}]),
            (31053, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "66205", "name": "u5916u89c2u65f6u5c1a", "num": "335"}, {"id": "33857", "name": "u4e58u5750u7a7au95f4u4e0du9519", "num": "328"}, {"id": "49912", "name": "u8f66u578bu914du7f6eu975eu5e38u6ee1u610f", "num": "192"}, {"id": "163586", "name": "u8f6cu5411u5f88u7cbeu51c6", "num": "138"}, {"id": "97291", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "134"}, {"id": "31991", "name": "u8f66u5185u9759u97f3u6548u679cu4e0du9519", "num": "114"}, {"id": "121199", "name": "u60acu67b6u51cfu9707u6709u70b9u786c", "num": "65"}, {"id": "209049", "name": "u540eu6392u7a7au95f4u6bd4u8f83u5c0f", "num": "37"}, {"id": "99913", "name": "u8d77u6b65u52a8u529bu6709u70b9u8089", "num": "35"}, {"id": "55467", "name": "u5012u8f66u5f71u50cfu592au5783u573e", "num": "33"}]),
            (31058, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "358164", "name": "u4e58u5750u7a7au95f4u5f88u559cu6b22", "num": "8"}]),
            (31464, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "553250", "name": "u6cb9u8017u6ee1u610f", "num": "25"}, {"id": "285410", "name": "u4e58u5750u7a7au95f4u591fu7528", "num": "16"}, {"id": "177927", "name": "u5916u89c2u6bd4u8f83u8f7b", "num": "16"}, {"id": "443034", "name": "u9694u97f3u4e0du597d", "num": "7"}, {"id": "476969", "name": "u8f6cu5411u975eu5e38u7075u6d3b", "num": "6"}, {"id": "314027", "name": "u5185u9970u9020u578bu4e0du597d", "num": "6"}, {"id": "151200", "name": "u8f66u5934u8bbeu8ba1u5f88u65f6u5c1a", "num": "5"}, {"id": "533572", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "4"}, {"id": "311299", "name": "u5473u9053u5f88u5c0f", "num": "4"}, {"id": "421662", "name": "u60acu67b6u51cfu9707u786c", "num": "3"}]),
            (31477, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "585652", "name": "u4e58u5750u7a7au95f4u591fu7528", "num": "5"}, {"id": "602441", "name": "u6c1bu56f4u706fu5f88u597d", "num": "5"}, {"id": "584512", "name": "u540eu6392u7a7au95f4u5f88u5927", "num": "3"}, {"id": "607139", "name": "u5916u89c2u7a33u91cd", "num": "3"}]),
            (31523, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "227853", "name": "u4e58u5750u7a7au95f4u51fau8272", "num": "28"}, {"id": "624528", "name": "u6cb9u8017u6ee1u610f", "num": "26"}, {"id": "522030", "name": "u5916u89c2u6f02u4eae", "num": "23"}, {"id": "640509", "name": "u8f6cu5411u624bu611fu5f88u8f7b", "num": "12"}, {"id": "409555", "name": "u52a8u529bu5341u8db3", "num": "11"}, {"id": "196527", "name": "u50a8u7269u7a7au95f4u5c11u7684u53efu601c", "num": "10"}, {"id": "647013", "name": "u9694u97f3u597d", "num": "9"}, {"id": "572994", "name": "u5473u9053u5f88u523au9f3b", "num": "9"}, {"id": "314967", "name": "u5185u9970u9020u578bu843du4f0d", "num": "4"}, {"id": "426904", "name": "u8f66u8eabu6296u52a8", "num": "3"}]),
            (31594, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "530401", "name": "u5916u89c2u4e0du9519", "num": "9"}, {"id": "487478", "name": "u4e58u5750u7a7au95f4u591fu7528", "num": "5"}, {"id": "524760", "name": "u9694u97f3u5f88u5f3a", "num": "5"}, {"id": "441959", "name": "u50a8u7269u7a7au95f4u591fu7528", "num": "4"}, {"id": "487671", "name": "u6cb9u8017u6ee1u610f", "num": "4"}, {"id": "507716", "name": "u4e2du63a7u5c4fu5f88u6e05", "num": "4"}, {"id": "499995", "name": "u540eu5907u53a2u7a7au95f4u5927", "num": "3"}, {"id": "455214", "name": "u8f66u5c3eu706fu4e0du592au4e60u60ef", "num": "3"}, {"id": "492401", "name": "u5473u9053u6bd4u8f83u5927", "num": "3"}]),
            (31841, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "136787", "name": "u6cb9u8017u6ee1u610f", "num": "23"}, {"id": "13165", "name": "u50a8u7269u7a7au95f4u5c11", "num": "17"}, {"id": "79390", "name": "u64cdu63a7u4e4bu738b", "num": "14"}, {"id": "104301", "name": "u80ceu566au7565u5927", "num": "13"}, {"id": "69852", "name": "u8f6cu5411u7cbeu51c6", "num": "12"}, {"id": "89225", "name": "u97f3u54cdu4e0du591f", "num": "12"}, {"id": "81020", "name": "u5916u89c2u9738u6c14", "num": "11"}, {"id": "150111", "name": "u8defu611fu53cdu9988u6e05u6670", "num": "8"}, {"id": "107009", "name": "u540eu6392u7a7au95f4u4e00u822cu822c", "num": "8"}, {"id": "65816", "name": "u52a0u901fu52a8u529bu5f88u5f3a", "num": "7"}]),
            (31921, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "283067", "name": "u4e58u5750u7a7au95f4u5341u8db3", "num": "13"}, {"id": "384203", "name": "u8f6cu5411u624bu611fu5f88u8f7b", "num": "7"}, {"id": "376286", "name": "u6cb9u8017u6ee1u610f", "num": "6"}, {"id": "299683", "name": "u6c1bu56f4u706fu5b8cu80dc", "num": "5"}, {"id": "216468", "name": "u60acu67b6u5f88u8212u670d", "num": "4"}, {"id": "215207", "name": "u5e95u76d8u60acu67b6u624eu5b9e", "num": "3"}, {"id": "330900", "name": "u9694u97f3u8db3u591f", "num": "3"}, {"id": "352408", "name": "u5916u89c2u7a33u91cd", "num": "3"}, {"id": "331775", "name": "u8f6cu5411u6709u865au4f4d", "num": "3"}, {"id": "348288", "name": "u97f3u54cdu4e00u822cu822c", "num": "3"}]),
            (32015, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "59072", "name": "u4e58u5750u7a7au95f4u5f88u6ee1u610f", "num": "72"}, {"id": "59301", "name": "u5916u89c2u6f02u4eae", "num": "54"}, {"id": "10245", "name": "u8f66u5b50u914du7f6eu6bd4u8f83u4e30u5bcc", "num": "45"}, {"id": "181618", "name": "u6cb9u8017u6ee1u610f", "num": "41"}, {"id": "181339", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "25"}, {"id": "181217", "name": "u5185u9970u9020u578bu5f88u597d", "num": "22"}, {"id": "174396", "name": "u8f6cu5411u624bu611fu4e0du51c6", "num": "12"}, {"id": "59520", "name": "u8d77u6b65u52a8u529bu8089", "num": "11"}, {"id": "181390", "name": "u5473u9053u6bd4u8f83u5927", "num": "10"}, {"id": "101632", "name": "u80ceu566au6bd4u8f83u5927", "num": "7"}]),
            (32064, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "59072", "name": "u4e58u5750u7a7au95f4u5f88u6ee1u610f", "num": "72"}, {"id": "59301", "name": "u5916u89c2u6f02u4eae", "num": "54"}, {"id": "10245", "name": "u8f66u5b50u914du7f6eu6bd4u8f83u4e30u5bcc", "num": "45"}, {"id": "181618", "name": "u6cb9u8017u6ee1u610f", "num": "41"}, {"id": "181339", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "25"}, {"id": "181217", "name": "u5185u9970u9020u578bu5f88u597d", "num": "22"}, {"id": "174396", "name": "u8f6cu5411u624bu611fu4e0du51c6", "num": "12"}, {"id": "59520", "name": "u8d77u6b65u52a8u529bu8089", "num": "11"}, {"id": "181390", "name": "u5473u9053u6bd4u8f83u5927", "num": "10"}, {"id": "101632", "name": "u80ceu566au6bd4u8f83u5927", "num": "7"}]),
            (32165, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "649841", "name": "u6cb9u8017u6ee1u610f", "num": "3"}]),
            (32168, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "661251", "name": "u5916u89c2u5b9eu7528", "num": "5"}, {"id": "667645", "name": "u4e58u5750u7a7au95f4u5b8cu5168u591f", "num": "4"}, {"id": "674742", "name": "u6cb9u8017u6ee1u610f", "num": "4"}, {"id": "666657", "name": "u8f6cu5411u624bu611fu5f88u7d27", "num": "4"}, {"id": "671927", "name": "u540eu5907u53a2u7a7au95f4u591fu7528u4e86", "num": "3"}, {"id": "669213", "name": "u64cdu63a7u5f88u68d2", "num": "3"}, {"id": "667147", "name": "u8f66u5c3eu706fu8ff7u4eba", "num": "3"}]),
            (32190, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "26245", "name": "u4e58u5750u7a7au95f4u591fu7528", "num": "190"}, {"id": "26877", "name": "u5916u89c2u6f02u4eae", "num": "182"}, {"id": "26953", "name": "u5b89u5168u914du7f6eu5f88u9ad8", "num": "172"}, {"id": "27186", "name": "u6cb9u8017u6ee1u610f", "num": "167"}, {"id": "26599", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "73"}, {"id": "27463", "name": "u8f6cu5411u624bu611fu5f88u8f7b", "num": "73"}, {"id": "26688", "name": "u5473u9053u5f88u5927", "num": "24"}, {"id": "235711", "name": "u5185u9970u6750u8d28u786c", "num": "23"}, {"id": "399275", "name": "u80ceu566au6bd4u8f83u660eu663e", "num": "19"}, {"id": "138468", "name": "u4e2du63a7u53f0u6750u8d28u6709u4e9bu5ec9u4ef7", "num": "11"}]),
            (32334, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "303231", "name": "u5916u89c2u5f88u65f6u5c1a", "num": "9"}, {"id": "366160", "name": "u4e58u5750u7a7au95f4u591fu7528", "num": "8"}, {"id": "581853", "name": "u6cb9u8017u6ee1u610f", "num": "8"}, {"id": "484371", "name": "u60acu67b6u51cfu9707u592au786c", "num": "6"}, {"id": "367600", "name": "u540eu5907u53a2u7a7au95f4u5927", "num": "3"}, {"id": "151960", "name": "u524du6392u5ea7u6905u5f88u4e0du9519", "num": "3"}, {"id": "195936", "name": "u8d77u6b65u52a8u529bu8089", "num": "3"}]),
            (32456, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "19375", "name": "u4e58u5750u7a7au95f4u4e0du9519", "num": "110"}, {"id": "144335", "name": "u5916u89c2u8010u770b", "num": "61"}, {"id": "243879", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "49"}, {"id": "19600", "name": "u5b89u5168u914du7f6eu6bd4u8f83u597d", "num": "43"}, {"id": "113701", "name": "u540eu6392u7a7au95f4u5f88u5927", "num": "27"}, {"id": "162997", "name": "u8f6cu5411u7cbeu51c6", "num": "27"}, {"id": "51260", "name": "u8d77u6b65u52a8u529bu8089", "num": "26"}, {"id": "19708", "name": "u80ceu566au592au5927", "num": "19"}, {"id": "115308", "name": "u5473u9053u6bd4u8f83u5927", "num": "16"}, {"id": "78648", "name": "u60acu67b6u51cfu9707u504fu786c", "num": "15"}]),
            (32464, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "55936", "name": "u6cb9u8017u6ee1u610f", "num": "45"}, {"id": "13628", "name": "u4e58u5750u7a7au95f4u4e0du6324", "num": "32"}, {"id": "37403", "name": "u5916u89c2u65f6u5c1a", "num": "30"}, {"id": "55816", "name": "u540eu5907u53a2u7a7au95f4u5927", "num": "28"}, {"id": "73270", "name": "u8f6cu5411u7cbeu51c6", "num": "16"}, {"id": "25783", "name": "u8d77u6b65u52a8u529bu8089", "num": "16"}, {"id": "80364", "name": "u60acu67b6u504fu786c", "num": "12"}, {"id": "42573", "name": "u64cdu63a7u4e0du9519", "num": "11"}, {"id": "55826", "name": "u540eu6392u7a7au95f4u6709u70b9u5c0f", "num": "7"}, {"id": "8550", "name": "u5473u9053u5927", "num": "6"}]),
            (32537, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "325680", "name": "u4e58u5750u7a7au95f4u6ca1u95eeu9898", "num": "104"}, {"id": "556052", "name": "u5916u89c2u9738u6c14", "num": "96"}, {"id": "366700", "name": "u8f66u5185u9759u97f3u6548u679cu5f88u597d", "num": "56"}, {"id": "356924", "name": "u5b89u5168u914du7f6eu826fu5fc3", "num": "52"}, {"id": "497469", "name": "u540eu5907u53a2u7a7au95f4u5927", "num": "44"}, {"id": "285501", "name": "u8f6cu5411u624bu611fu6bd4u8f83u8f7b", "num": "40"}, {"id": "645045", "name": "u97f3u54cdu6548u679cu5f88u68d2", "num": "33"}, {"id": "313968", "name": "u5012u8f66u5f71u50cfu96beu53d7", "num": "7"}, {"id": "528839", "name": "u53d1u52a8u673au542fu505cu5751", "num": "4"}, {"id": "600014", "name": "u5012u8f66u96f7u8fbeu5faeu4e0du8db3u9053", "num": "3"}]),
            (32567, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "254715", "name": "u5916u89c2u9738u6c14", "num": "92"}, {"id": "253262", "name": "u4e58u5750u7a7au95f4u975eu5e38u4e0du9519", "num": "75"}, {"id": "766351", "name": "u4e58u5750u7a7au95f4u975eu5e38u5bbdu655e", "num": "31"}, {"id": "540417", "name": "u6cb9u8017u4e0du51c6", "num": "22"}, {"id": "625267", "name": "u56dbu9a71u7cfbu7edfu5b8cu7f8e", "num": "19"}, {"id": "255716", "name": "u8f6cu5411u5f88u7075u6d3b", "num": "19"}, {"id": "766655", "name": "u6cb9u8017u4e0du4f4e", "num": "19"}, {"id": "562969", "name": "u50a8u7269u7a7au95f4u6bd4u8f83u4e30u5bcc", "num": "17"}, {"id": "734958", "name": "u60acu67b6u6709u70b9u786c", "num": "11"}, {"id": "772272", "name": "u5ea7u6905u8bbeu8ba1u6709u70b9u786c", "num": "7"}]),
            (32700, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "538295", "name": "u4e58u5750u7a7au95f4u4f18u79c0", "num": "5"}, {"id": "475853", "name": "u6cb9u8017u60f3u8c61u4e2du597d", "num": "4"}]),
            (33034, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "655334", "name": "u5916u89c2u6d41u7545", "num": "5"}]),
            (33068, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "152426", "name": "u4e58u5750u7a7au95f4u5f88u4e0du9519", "num": "4"}, {"id": "568102", "name": "u60acu67b6u652fu6491u7a33u5b9a", "num": "3"}]),
            (33279, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "145946", "name": "u6cb9u8017u6ee1u610f", "num": "38"}, {"id": "108671", "name": "u4e58u5750u7a7au95f4u86eeu4e0du9519", "num": "34"}, {"id": "769245", "name": "u4e58u5750u7a7au95f4u4e0du9519", "num": "12"}, {"id": "104744", "name": "u540eu6392u7a7au95f4u5f88u8db3", "num": "9"}, {"id": "46445", "name": "u5916u89c2u65f6u5c1a", "num": "9"}, {"id": "15790", "name": "u9694u97f3u5dee", "num": "8"}, {"id": "64668", "name": "u5b89u5168u914du7f6eu5deeu4e86u70b9", "num": "8"}, {"id": "10795", "name": "u50a8u7269u7a7au95f4u591fu7528", "num": "7"}, {"id": "58449", "name": "u8f6cu5411u504fu6c89", "num": "7"}, {"id": "43472", "name": "u60acu67b6u6bd4u8f83u786c", "num": "5"}]),
            (33307, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "426440", "name": "u4e58u5750u7a7au95f4u5927", "num": "6"}]),
            (33320, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "13213", "name": "u4e58u5750u7a7au95f4u8212u9002", "num": "128"}, {"id": "202734", "name": "u6cb9u8017u6ee1u610f", "num": "126"}, {"id": "54686", "name": "u5916u89c2u5f88u6f02u4eae", "num": "41"}, {"id": "13233", "name": "u524du6392u5ea7u6905u5f88u8212u670d", "num": "31"}, {"id": "13238", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "30"}, {"id": "29103", "name": "u8d77u6b65u52a8u529bu8089", "num": "30"}, {"id": "13264", "name": "u540eu6392u7a7au95f4u5f88u5927", "num": "27"}, {"id": "181684", "name": "u5916u6f06u9762u592au8584u4e86", "num": "24"}, {"id": "37468", "name": "u50a8u7269u7a7au95f4u592au5c11", "num": "23"}, {"id": "10138", "name": "u5473u9053u5927", "num": "23"}]),
            (33339, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "347070", "name": "u4e58u5750u7a7au95f4u6bd4u8f83u6ee1u610f", "num": "36"}, {"id": "536193", "name": "u5916u89c2u6f02u4eae", "num": "23"}, {"id": "405400", "name": "u8f6cu5411u8f7bu76c8", "num": "9"}, {"id": "340625", "name": "u9ad8u901fu6cb9u8017u6ee1u610f", "num": "8"}, {"id": "643534", "name": "u8f6cu5411u624bu611fu5f88u8f7b", "num": "7"}, {"id": "571362", "name": "u50a8u7269u7a7au95f4u4e30u5bcc", "num": "6"}, {"id": "634207", "name": "u53d1u52a8u673au566au97f3u633au5927", "num": "5"}, {"id": "447141", "name": "u5473u9053u6bd4u8f83u5927", "num": "5"}, {"id": "503401", "name": "u60acu67b6u504fu786c", "num": "4"}, {"id": "244673", "name": "u98ceu566au5f88u5927", "num": "3"}]),
            (33361, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "121848", "name": "u4e58u5750u7a7au95f4u8fd8u4e0du9519", "num": "45"}, {"id": "144105", "name": "u5916u89c2u6f02u4eae", "num": "39"}, {"id": "75324", "name": "u5b89u5168u914du7f6eu5f88u597d", "num": "34"}, {"id": "228340", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "20"}, {"id": "160931", "name": "u60acu67b6u51cfu9707u786c", "num": "19"}, {"id": "17467", "name": "u5185u9970u9020u578bu975eu5e38u4e0au6863u6b21", "num": "16"}, {"id": "108317", "name": "u8f6cu5411u7cbeu51c6", "num": "14"}, {"id": "793422", "name": "u60acu67b6u51cfu9707u504fu786c", "num": "5"}, {"id": "89289", "name": "u8d77u6b65u52a8u529bu6709u70b9u8089", "num": "4"}, {"id": "185419", "name": "u98ceu566au8f83u5927", "num": "3"}]),
            (33595, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "362087", "name": "u4e58u5750u7a7au95f4u6ca1u95eeu9898", "num": "6"}, {"id": "569377", "name": "u5916u89c2u9738u6c14", "num": "4"}]),
            (33999, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "335637", "name": "u6cb9u8017u6ee1u610f", "num": "20"}, {"id": "25411", "name": "u4e58u5750u7a7au95f4u591fu7528", "num": "13"}, {"id": "25510", "name": "u5916u89c2u4e0du9519", "num": "8"}, {"id": "102684", "name": "u540eu5907u53a2u7a7au95f4u633au5927", "num": "7"}, {"id": "767145", "name": "u4e58u5750u7a7au95f4u4e0du9519", "num": "6"}, {"id": "143418", "name": "u540eu6392u7a7au95f4u633au5bbdu655e", "num": "5"}, {"id": "317386", "name": "u5b9au901fu5de1u822au975eu5e38u597du7528", "num": "4"}, {"id": "176510", "name": "u4e2du9014u52a0u901fu6bd4u8f83u5feb", "num": "3"}, {"id": "225554", "name": "u8f6cu5411u7075u6d3b", "num": "3"}, {"id": "712064", "name": "u8d77u6b65u52a8u529bu8089", "num": "3"}]),
            (34033, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "706583", "name": "u6cb9u8017u6ee1u610f", "num": "27"}, {"id": "520739", "name": "u4e58u5750u7a7au95f4u8db3u591fu7528", "num": "22"}, {"id": "664360", "name": "u5916u89c2u6f02u4eae", "num": "17"}, {"id": "766782", "name": "u5916u89c2u9ad8u7aefu5927u6c14u4e0au6863u6b21", "num": "16"}, {"id": "777053", "name": "u4e58u5750u7a7au95f4u8db3u591f", "num": "12"}, {"id": "519492", "name": "u8f66u5185u9759u97f3u6548u679cu4e0du597d", "num": "9"}, {"id": "703029", "name": "u540eu5907u53a2u7a7au95f4u5f88u5927", "num": "8"}, {"id": "713387", "name": "u8d77u6b65u52a8u529bu6709u70b9u8089", "num": "4"}, {"id": "369619", "name": "u53d1u52a8u673au566au97f3u592au5927", "num": "4"}, {"id": "431237", "name": "u80ceu566au6bd4u8f83u5927", "num": "4"}]),
            (34127, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "424814", "name": "u8f6cu5411u7cbeu51c6", "num": "3"}]),
            (34169, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "32276", "name": "u4e58u5750u7a7au95f4u8db3u591f", "num": "42"}, {"id": "767268", "name": "u4e58u5750u7a7au95f4u6ca1u7684u8bf4", "num": "32"}, {"id": "96588", "name": "u8f6cu5411u7cbeu51c6", "num": "20"}, {"id": "157443", "name": "u5916u89c2u4e0du9519", "num": "17"}, {"id": "770123", "name": "u5916u89c2u65f6u5c1a", "num": "16"}, {"id": "84062", "name": "u8d77u6b65u52a8u529bu6709u70b9u8089", "num": "16"}, {"id": "90214", "name": "u540eu5907u53a2u7a7au95f4u5927", "num": "14"}, {"id": "4223", "name": "u80ceu566au4e0du5c0f", "num": "10"}, {"id": "767348", "name": "u60acu67b6u504fu786c", "num": "7"}, {"id": "124351", "name": "u60acu67b6u786c", "num": "7"}]),
            (34196, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "731438", "name": "u6cb9u8017u8fd8u7b97u53efu4ee5", "num": "5"}, {"id": "767308", "name": "u5b89u5168u914du7f6eu5f88u653eu5fc3", "num": "4"}, {"id": "731038", "name": "u540eu5907u53a2u7a7au95f4u6bd4u8f83u5927", "num": "3"}]),
            (34276, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "413667", "name": "u4e58u5750u7a7au95f4u76f8u5f53u4e0du9519", "num": "4"}, {"id": "414034", "name": "u5916u89c2u5f88u6f02u4eae", "num": "3"}]),
            (34764, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "756720", "name": "u6cb9u8017u6ee1u610f", "num": "5"}, {"id": "770325", "name": "u4e58u5750u7a7au95f4u6ca1u95eeu9898", "num": "3"}]),
            (1002001, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "361258", "name": "u4e58u5750u7a7au95f4u591fu7528", "num": "22"}, {"id": "345302", "name": "u9694u97f3u4e0du9519", "num": "6"}, {"id": "322520", "name": "u540eu6392u7a7au95f4u5927", "num": "5"}, {"id": "347283", "name": "u97f3u54cdu4e0du9519", "num": "4"}, {"id": "311781", "name": "u5185u9970u9020u578bu633au6ee1u610f", "num": "3"}, {"id": "365843", "name": "u5927u706fu4e0du4eae", "num": "3"}]),
            (1003454, [{"id": "0", "name": "u5168u90e8", "num": ""}, {"id": "555567", "name": "u4e58u5750u7a7au95f4u6bd4u8f83u591fu7528", "num": "39"}, {"id": "607786", "name": "u6cb9u8017u6ee1u610f", "num": "34"}, {"id": "474500", "name": "u5916u89c2u6f02u4eae", "num": "19"}, {"id": "452827", "name": "u8f6cu5411u6709u865au4f4d", "num": "7"}, {"id": "533829", "name": "u9694u97f3u4e0du597d", "num": "7"}, {"id": "419240", "name": "u524du6392u5ea7u6905u8212u670d", "num": "5"}, {"id": "374923", "name": "u5185u9970u9020u578bu6ee1u610f", "num": "5"}, {"id": "410122", "name": "u97f3u54cdu4e0du9519", "num": "5"}, {"id": "494642", "name": "u60acu67b6u51cfu9707u786c", "num": "5"}, {"id": "514133", "name": "u5916u6f06u9762u592au8584", "num": "5"}]),
        ]

        for (gid, tag_li2) in tag_li:
            # gid = '1002001'
            # tagid = '361258'
            for tag in tag_li2:
                tagid = tag.get('id', '0')
                if tagid=='0':
                    continue
                goods = {
                    'gid': gid,
                    'tagid': tagid,
                }
                header['Referer'] = 'https://m.autohome.com.cn/spec/%s' % (gid, )
                url = 'https://k.m.autohome.com.cn/spec/%s/?min_p=%d' % (gid, 1)
                js_script = """
                    function() {
                        setTimeout(function() {
                            for (var i=0,len=$('.tabSelect').length; i<len; i++) {
                                tagid = $($('.tabSelect')[i]).attr('value')
                                if(tagid=='%s') {
                                    $('.tabSelect')[i].click()
                                    break;
                                }
                                console.log(tagid)
                            }
                        }, 300);
                    }
                """ % (tagid, )
                self.crawl(url, callback=self.index_page, headers=header, cookies=self.get_cookies(), save=goods, fetch_type='js', js_script=js_script)

    @config(age=10 * 24 * 60 * 60)
    def index_page(self, response):
        self.crawl_config['proxy'] = get_proxy()
        header_slt = HeadersSelector()
        header = header_slt.select_header()  # 获取一个新的 header

        header['Host'] = 'k.m.autohome.com.cn'
        doc = response.doc
        save = response.save
        gid = save.get('gid', '0')
        tagid = save.get('tagid', '0')

        (min_p, max_p) = save.get('min_max', [0,0])
        min_p = int(min_p)
        max_p = int(max_p)
        #comments = save.get('comments', [])
        #print('comments', comments)
        try:
            # print(doc.html())
            if min_p==0:
                pattern = re.compile(r'pageNumber.*val\((\d)+.*(\d)+\)', re.U)
                [(min_p, max_p)] = pattern.findall(doc.html())
                min_p = int(min_p)
                max_p = int(max_p)
            print(min_p, max_p, max_p>min_p)
            print(doc('#js-pageNumber').val())

            comments2 = ['https:%s' % comm.attr('onclick')[26:-1] for comm in doc('.comments ul').items()]
            if comments2:
                js_script = """
                    function() {
                        setTimeout(function() {
                        }, 3000);
                    }
                """
                for url in comments2:
                    yield self.crawl(url, callback=self.detail_page, save=save, headers=header, fetch_type='js', js_script=js_script)

            #save['comments'] = list(set(comments + comments2))
            #save['comments'] = comments + comments2
            #print("save['comments']", save['comments'])
            if max_p>min_p:
                save['min_max'] = [min_p+1, max_p]
                js_script = """
                    function() {
                        setTimeout(function() {
                            $("#js-pageNumber").val(%d + '/' + %d);
                            $('#pageNext').click();
                        }, 300);
                    }
                """ % (min_p+1, max_p)

                url = 'https://k.m.autohome.com.cn/spec/%s/?min_p=%d' % (gid, min_p+1)
                self.crawl(response.url, callback=self.index_page, save=save, fetch_type='js', js_script=js_script)
        except Exception as e:
            raise e
            pass

    @config(priority=2)
    def detail_page(self, response):
        doc = response.doc
        save = response.save
        gid = save.get('gid', '0')
        tagid = save.get('tagid', '0')

        comment = {}
        comment['url'] = response.url
        comment['goods_id'] = gid
        comment['goods_name'] = doc('.cartype a').text()
        comment['title'] = doc('h1').text()
        comment['category'] = ''
        comment['tagid'] = tagid
        comment['author'] = doc('.wom-details .info .name span').text()
        comment['pubtime'] = doc('.wom-details .info time').text()
        comment['dealer'] = doc('.site').text()
        comment['buy_date'] = doc('.date span').text()
        items = list(doc('.unit-gather .item').items())
        if not items:
            items = list(doc('.edition-base .unit-gather .item').items())
        #print(items, comment)
        comment['naked_car_price'] = items[0].text()
        comment['oil_wear'] = items[1].text()
        comment['current_drive'] = items[2].text()
        comment['content'] = html.escape(doc('.matter').html())
        return comment

    def on_result(self,result):
        print(result)
        if not result or not result.get('goods_id', ''):
            return
        sql = SQL()
        header = result.get('header', None)
        if header:
            result.pop('header')
        # print(result)
        sql.replace('jjb_goods_car_comment', result)
